home *** CD-ROM | disk | FTP | other *** search
/ Personal Computer World 2008 February / PCWFEB08.iso / Software / Freeware / Miro 1.0 / Miro_Installer.exe / Miro_Downloader.exe / httpclient.pyc (.txt) < prev    next >
Encoding:
Python Compiled Bytecode  |  2007-11-12  |  65.6 KB  |  2,108 lines
  1. # Source Generated with Decompyle++
  2. # File: in.pyc (Python 2.5)
  3.  
  4. """httpclient.py 
  5.  
  6. Implements an HTTP client.  The main way that this module is used is the
  7. grabURL function that's an asynchronous version of our old grabURL.
  8.  
  9. A lot of the code here comes from inspection of the httplib standard module.
  10. Some of it was taken more-or-less directly from there.  I (Ben Dean-Kawamura)
  11. believe our clients follow the HTTP 1.1 spec completely, I used RFC2616 as a
  12. reference (http://www.w3.org/Protocols/rfc2616/rfc2616.html).
  13. """
  14. import errno
  15. import logging
  16. import re
  17. import socket
  18. import traceback
  19. from urlparse import urljoin
  20. from gtcache import gettext as _
  21. from base64 import b64encode
  22. from clock import clock
  23. import httpauth
  24. import config
  25. import prefs
  26. from download_utils import URIPattern, cleanFilename, parseURL, defaultPort, getFileURLPath, filenameFromURL
  27. from xhtmltools import URLEncodeDict, multipartEncode
  28. import eventloop
  29. import util
  30. import sys
  31. import time
  32. import urllib
  33. PIPELINING_ENABLED = True
  34. SOCKET_READ_TIMEOUT = 60
  35. SOCKET_INITIAL_READ_TIMEOUT = 30
  36. SOCKET_CONNECT_TIMEOUT = 15
  37.  
  38. class NetworkError(Exception):
  39.     '''Base class for all errors that will be passed to errbacks from getURL
  40.     and friends.  NetworkErrors can be display in 2 ways:
  41.  
  42.     getFriendlyDescription() -- short, newbie friendly description 
  43.     getLongDescription() -- detailed description
  44.     '''
  45.     
  46.     def __init__(self, shortDescription, longDescription = None):
  47.         if longDescription is None:
  48.             longDescription = shortDescription
  49.         
  50.         self.friendlyDescription = _('Error: %s') % shortDescription
  51.         self.longDescription = longDescription
  52.  
  53.     
  54.     def getFriendlyDescription(self):
  55.         return self.friendlyDescription
  56.  
  57.     
  58.     def getLongDescription(self):
  59.         return self.longDescription
  60.  
  61.     
  62.     def __str__(self):
  63.         return '%s: %s -- %s' % (self.__class__, self.getFriendlyDescription(), self.getLongDescription())
  64.  
  65.  
  66.  
  67. class ConnectionError(NetworkError):
  68.     
  69.     def __init__(self, errorMessage):
  70.         self.friendlyDescription = _("Can't connect")
  71.         self.longDescription = _('Connection Error: %s') % util.stringify(errorMessage, 'replace')
  72.  
  73.  
  74.  
  75. class SSLConnectionError(ConnectionError):
  76.     
  77.     def __init__(self):
  78.         self.friendlyDescription = _("Can't connect")
  79.         self.longDescription = _('SSL connection error')
  80.  
  81.  
  82.  
  83. class HTTPError(NetworkError):
  84.     
  85.     def __init__(self, longDescription):
  86.         NetworkError.__init__(self, _('HTTP error'), longDescription)
  87.  
  88.  
  89.  
  90. class BadStatusLine(HTTPError):
  91.     
  92.     def __init__(self, line):
  93.         HTTPError.__init__(self, _('Bad Status Line: %s') % line)
  94.  
  95.  
  96.  
  97. class BadHeaderLine(HTTPError):
  98.     
  99.     def __init__(self, line):
  100.         HTTPError.__init__(self, _('Bad Header Line: %s') % line)
  101.  
  102.  
  103.  
  104. class BadChunkSize(HTTPError):
  105.     
  106.     def __init__(self, line):
  107.         HTTPError.__init__(self, _('Bad Chunk size: %s') % line)
  108.  
  109.  
  110.  
  111. class CRLFExpected(HTTPError):
  112.     
  113.     def __init__(self, crlf):
  114.         HTTPError.__init__(self, _('Expected CRLF got: %r') % crlf)
  115.  
  116.  
  117.  
  118. class ServerClosedConnection(HTTPError):
  119.     
  120.     def __init__(self, host):
  121.         HTTPError.__init__(self, _('%s closed connection') % host)
  122.  
  123.  
  124.  
  125. class UnexpectedStatusCode(HTTPError):
  126.     
  127.     def __init__(self, code):
  128.         if code == 404:
  129.             self.friendlyDescription = _('File not found')
  130.             self.longDescription = _('Got 404 status code')
  131.         else:
  132.             HTTPError.__init__(self, _('Bad Status Code: %s') % code)
  133.  
  134.  
  135.  
  136. class AuthorizationFailed(NetworkError):
  137.     
  138.     def __init__(self):
  139.         NetworkError.__init__(self, _('Authorization failed'))
  140.  
  141.  
  142.  
  143. class PipelinedRequestNeverStarted(NetworkError):
  144.     
  145.     def __init__(self):
  146.         NetworkError.__init__(self, _('Internal Error'), _('Pipeline request never started'))
  147.  
  148.  
  149.  
  150. class ConnectionTimeout(NetworkError):
  151.     
  152.     def __init__(self, host):
  153.         NetworkError.__init__(self, _('Timeout'), _('Connection to %s timed out') % host)
  154.  
  155.  
  156.  
  157. class MalformedURL(NetworkError):
  158.     
  159.     def __init__(self, url):
  160.         NetworkError.__init__(self, _('Invalid URL'), _('"%s" is not a valid URL') % url)
  161.  
  162.  
  163.  
  164. class FileURLNotFoundError(NetworkError):
  165.     """A file: URL doesn't exist"""
  166.     
  167.     def __init__(self, path):
  168.         NetworkError.__init__(self, _('File not found'), _('The file: "%s" doesn\'t exist') % path)
  169.  
  170.  
  171.  
  172. class FileURLReadError(NetworkError):
  173.     
  174.     def __init__(self, path):
  175.         NetworkError.__init__(self, _('Read error'), _('Error while reading from "%s"') % path)
  176.  
  177.  
  178.  
  179. def trapCall(object, function, *args, **kwargs):
  180.     """Convenience function do a util.trapCall, where when = 'While talking to
  181.     the network'
  182.     """
  183.     return util.timeTrapCall('Calling %s on %s' % (function, object), function, *args, **kwargs)
  184.  
  185. DATEINFUTURE = time.mktime((2030, 7, 12, 12, 0, 0, 4, 193, -1))
  186.  
  187. def get_cookie_expiration_date(val):
  188.     """Tries a bunch of possible cookie expiration date formats
  189.     until it finds the magic one (or doesn't and returns 0).
  190.     """
  191.     fmts = ('%a, %d %b %Y %H:%M:%S %Z', '%a, %d %b %y %H:%M:%S %Z', '%a, %d-%b-%Y %H:%M:%S %Z', '%a, %d-%b-%y %H:%M:%S %Z')
  192.     for fmt in fmts:
  193.         
  194.         try:
  195.             return time.mktime(time.strptime(val, fmt))
  196.         continue
  197.         except OverflowError:
  198.             oe = None
  199.             return DATEINFUTURE
  200.             continue
  201.             except ValueError:
  202.                 ve = None
  203.                 continue
  204.             
  205.         print "DTV: Warning: Can't process cookie expiration: '%s'" % val
  206.         return 0
  207.  
  208.  
  209.  
  210. class NetworkBuffer(object):
  211.     '''Responsible for storing incomming network data and doing some basic
  212.     parsing of it.  I think this is about as fast as we can do things in pure
  213.     python, someday we may want to make it C...
  214.     '''
  215.     
  216.     def __init__(self):
  217.         self.chunks = []
  218.         self.length = 0
  219.  
  220.     
  221.     def addData(self, data):
  222.         self.chunks.append(data)
  223.         self.length += len(data)
  224.  
  225.     
  226.     def _mergeChunks(self):
  227.         self.chunks = [
  228.             ''.join(self.chunks)]
  229.  
  230.     
  231.     def read(self, size = None):
  232.         '''Read at most size bytes from the data that has been added to the
  233.         buffer.  '''
  234.         self._mergeChunks()
  235.         if size is not None:
  236.             rv = self.chunks[0][:size]
  237.             self.chunks[0] = self.chunks[0][len(rv):]
  238.         else:
  239.             rv = self.chunks[0]
  240.             self.chunks = []
  241.         self.length -= len(rv)
  242.         return rv
  243.  
  244.     
  245.     def readline(self):
  246.         '''Like a file readline, with several difference:  
  247.         * If there isn\'t a full line ready to be read we return None.  
  248.         * Doesn\'t include the trailing line separator.
  249.         * Both "\r
  250. " and "
  251. " act as a line ender
  252.         '''
  253.         self._mergeChunks()
  254.         split = self.chunks[0].split('\n', 1)
  255.         if len(split) == 2:
  256.             self.chunks[0] = split[1]
  257.             self.length = len(self.chunks[0])
  258.             if split[0].endswith('\r'):
  259.                 return split[0][:-1]
  260.             else:
  261.                 return split[0]
  262.         else:
  263.             return None
  264.  
  265.     
  266.     def unread(self, data):
  267.         '''Put back read data.  This make is like the data was never read at
  268.         all.
  269.         '''
  270.         self.chunks.insert(0, data)
  271.         self.length += len(data)
  272.  
  273.     
  274.     def getValue(self):
  275.         self._mergeChunks()
  276.         return self.chunks[0]
  277.  
  278.  
  279.  
  280. class _Packet(object):
  281.     '''A packet of data for the AsyncSocket class
  282.     '''
  283.     
  284.     def __init__(self, data, callback = None):
  285.         self.data = data
  286.         self.callback = callback
  287.  
  288.  
  289.  
  290. class AsyncSocket(object):
  291.     '''Socket class that uses our new fangled asynchronous eventloop
  292.     module.
  293.     '''
  294.     MEMORY_ERROR_LIMIT = 5
  295.     
  296.     def __init__(self, closeCallback = None):
  297.         '''Create an AsyncSocket.  If closeCallback is given, it will be
  298.         called if we detect that the socket has been closed durring a
  299.         read/write operation.  The arguments will be the AsyncSocket object
  300.         and either socket.SHUT_RD or socket.SHUT_WR.
  301.         '''
  302.         self.toSend = []
  303.         self.readSize = 4096
  304.         self.socket = None
  305.         self.readCallback = None
  306.         self.closeCallback = closeCallback
  307.         self.readTimeout = None
  308.         self.timedOut = False
  309.         self.connectionErrback = None
  310.         self.disableReadTimeout = False
  311.         self.readSomeData = False
  312.         self.name = ''
  313.         self.lastClock = None
  314.         self.memoryErrors = 0
  315.  
  316.     
  317.     def __str__(self):
  318.         if self.name:
  319.             return '%s: %s' % (type(self).__name__, self.name)
  320.         else:
  321.             return 'Unknown %s' % (type(self).__name__,)
  322.  
  323.     
  324.     def startReadTimeout(self):
  325.         if self.disableReadTimeout:
  326.             return None
  327.         
  328.         self.lastClock = clock()
  329.         if self.readTimeout is not None:
  330.             return None
  331.         
  332.         self.readTimeout = eventloop.addTimeout(SOCKET_INITIAL_READ_TIMEOUT, self.onReadTimeout, 'AsyncSocket.onReadTimeout')
  333.  
  334.     
  335.     def stopReadTimeout(self):
  336.         if self.readTimeout is not None:
  337.             self.readTimeout.cancel()
  338.             self.readTimeout = None
  339.         
  340.  
  341.     
  342.     def openConnection(self, host, port, callback, errback, disableReadTimeout = None):
  343.         '''Open a connection.  On success, callback will be called with this
  344.         object.
  345.         '''
  346.         if disableReadTimeout is not None:
  347.             self.disableReadTimeout = disableReadTimeout
  348.         
  349.         self.name = 'Outgoing %s:%s' % (host, port)
  350.         
  351.         try:
  352.             self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  353.         except socket.error:
  354.             e = None
  355.             trapCall(self, errback, ConnectionError(e[1]))
  356.             return None
  357.  
  358.         self.socket.setblocking(0)
  359.         self.connectionErrback = errback
  360.         
  361.         def handleGetHostByNameException(e):
  362.             trapCall(self, errback, ConnectionError(e[1]))
  363.  
  364.         
  365.         def onAddressLookup(address):
  366.             if self.socket is None:
  367.                 return None
  368.             
  369.             
  370.             try:
  371.                 rv = self.socket.connect_ex((address, port))
  372.             except socket.gaierror:
  373.                 trapCall(self, errback, ConnectionError('gaierror'))
  374.                 return None
  375.  
  376.             if rv in (0, errno.EINPROGRESS, errno.EWOULDBLOCK):
  377.                 eventloop.addWriteCallback(self.socket, onWriteReady)
  378.                 self.socketConnectTimeout = eventloop.addTimeout(SOCKET_CONNECT_TIMEOUT, onWriteTimeout, 'socket connect timeout')
  379.             else:
  380.                 msg = errno.errorcode[rv]
  381.                 trapCall(self, errback, ConnectionError(msg))
  382.  
  383.         
  384.         def onWriteReady():
  385.             eventloop.removeWriteCallback(self.socket)
  386.             self.socketConnectTimeout.cancel()
  387.             rv = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR)
  388.             if rv == 0:
  389.                 trapCall(self, callback, self)
  390.             else:
  391.                 msg = errno.errorcode.get(rv, _('Unknown Error code'))
  392.                 trapCall(self, errback, ConnectionError(msg))
  393.             self.connectionErrback = None
  394.  
  395.         
  396.         def onWriteTimeout():
  397.             eventloop.removeWriteCallback(self.socket)
  398.             trapCall(self, errback, ConnectionTimeout(host))
  399.             self.connectionErrback = None
  400.  
  401.         eventloop.callInThread(onAddressLookup, handleGetHostByNameException, socket.gethostbyname, 'getHostByName - %s' % host, host)
  402.  
  403.     
  404.     def acceptConnection(self, host, port, callback, errback):
  405.         
  406.         def finishAccept():
  407.             eventloop.removeReadCallback(self.socket)
  408.             (self.socket, addr) = self.socket.accept()
  409.             trapCall(self, callback, self)
  410.             self.connectionErrback = None
  411.  
  412.         self.name = 'Incoming %s:%s' % (host, port)
  413.         self.connectionErrback = errback
  414.         self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
  415.         self.socket.bind((host, port))
  416.         (self.addr, self.port) = self.socket.getsockname()
  417.         self.socket.listen(63)
  418.         eventloop.addReadCallback(self.socket, finishAccept)
  419.  
  420.     
  421.     def closeConnection(self):
  422.         if self.isOpen():
  423.             eventloop.stopHandlingSocket(self.socket)
  424.             self.stopReadTimeout()
  425.             self.socket.close()
  426.             self.socket = None
  427.             if self.connectionErrback is not None:
  428.                 error = NetworkError(_('Connection closed'))
  429.                 trapCall(self, self.connectionErrback, error)
  430.                 self.connectionErrback = None
  431.             
  432.         
  433.  
  434.     
  435.     def isOpen(self):
  436.         return self.socket is not None
  437.  
  438.     
  439.     def sendData(self, data, callback = None):
  440.         '''Send data out to the socket when it becomes ready.
  441.         
  442.         NOTE: currently we have no way of detecting when the data gets sent
  443.         out, or if errors happen.
  444.         '''
  445.         if not self.isOpen():
  446.             raise ValueError('Socket not connected')
  447.         
  448.         self.toSend.append(_Packet(data, callback))
  449.         eventloop.addWriteCallback(self.socket, self.onWriteReady)
  450.  
  451.     
  452.     def startReading(self, readCallback):
  453.         '''Start reading from the socket.  When data becomes available it will
  454.         be passed to readCallback.  If there is already a read callback, it
  455.         will be replaced.
  456.         '''
  457.         if not self.isOpen():
  458.             raise ValueError('Socket not connected')
  459.         
  460.         self.readCallback = readCallback
  461.         eventloop.addReadCallback(self.socket, self.onReadReady)
  462.         self.startReadTimeout()
  463.  
  464.     
  465.     def stopReading(self):
  466.         '''Stop reading from the socket.'''
  467.         if not self.isOpen():
  468.             raise ValueError('Socket not connected')
  469.         
  470.         self.readCallback = None
  471.         eventloop.removeReadCallback(self.socket)
  472.         self.stopReadTimeout()
  473.  
  474.     
  475.     def onReadTimeout(self):
  476.         if self.readSomeData:
  477.             timeout = SOCKET_READ_TIMEOUT
  478.         else:
  479.             timeout = SOCKET_INITIAL_READ_TIMEOUT
  480.         if clock() < self.lastClock + timeout:
  481.             self.readTimeout = eventloop.addTimeout(self.lastClock + timeout - clock(), self.onReadTimeout, 'AsyncSocket.onReadTimeout')
  482.         else:
  483.             self.readTimeout = None
  484.             self.timedOut = True
  485.             self.handleEarlyClose('read')
  486.  
  487.     
  488.     def handleSocketError(self, code, msg, operation):
  489.         if code in (errno.EWOULDBLOCK, errno.EINTR):
  490.             return None
  491.         
  492.         if operation == 'write':
  493.             expectedErrors = (errno.EPIPE, errno.ECONNRESET)
  494.         else:
  495.             expectedErrors = (errno.ECONNREFUSED, errno.ECONNRESET)
  496.         if code not in expectedErrors:
  497.             print 'WARNING, got unexpected error during %s' % operation
  498.             print '%s: %s' % (errno.errorcode.get(code), msg)
  499.         
  500.         self.handleEarlyClose(operation)
  501.  
  502.     
  503.     def onWriteReady(self):
  504.         
  505.         try:
  506.             if len(self.toSend) > 0:
  507.                 sent = self.socket.send(self.toSend[0].data)
  508.             else:
  509.                 sent = 0
  510.         except socket.error:
  511.             (code, msg) = None
  512.             self.handleSocketError(code, msg, 'write')
  513.  
  514.         self.handleSentData(sent)
  515.  
  516.     
  517.     def handleSentData(self, sent):
  518.         if len(self.toSend) > 0:
  519.             self.toSend[0].data = self.toSend[0].data[sent:]
  520.             if len(self.toSend[0].data) == 0:
  521.                 if self.toSend[0].callback:
  522.                     self.toSend[0].callback()
  523.                 
  524.                 self.toSend = self.toSend[1:]
  525.             
  526.         
  527.         if len(self.toSend) == 0:
  528.             eventloop.removeWriteCallback(self.socket)
  529.         
  530.  
  531.     
  532.     def onReadReady(self):
  533.         
  534.         try:
  535.             data = self.socket.recv(self.readSize)
  536.         except socket.error:
  537.             (code, msg) = None
  538.             self.handleSocketError(code, msg, 'read')
  539.         except MemoryError:
  540.             self.memoryErrors += 1
  541.             if self.memoryErrors > self.MEMORY_ERROR_LIMIT:
  542.                 print 'ERROR: Too many MemoryErrors on %s' % self
  543.                 self.handleEarlyClose('read')
  544.             else:
  545.                 print 'WARNING: Memory error while reading from %s' % self
  546.         except:
  547.             self.memoryErrors > self.MEMORY_ERROR_LIMIT
  548.  
  549.         self.memoryErrors = 0
  550.         self.handleReadData(data)
  551.  
  552.     
  553.     def handleReadData(self, data):
  554.         self.startReadTimeout()
  555.         if data == '':
  556.             if self.closeCallback:
  557.                 trapCall(self, self.closeCallback, self, socket.SHUT_RD)
  558.             
  559.         else:
  560.             self.readSomeData = True
  561.             trapCall(self, self.readCallback, data)
  562.  
  563.     
  564.     def handleEarlyClose(self, operation):
  565.         self.closeConnection()
  566.         if self.closeCallback:
  567.             if operation == 'read':
  568.                 type = socket.SHUT_RD
  569.             else:
  570.                 type = socket.SHUT_WR
  571.             trapCall(self, self.closeCallback, self, type)
  572.         
  573.  
  574.  
  575.  
  576. class AsyncSSLStream(AsyncSocket):
  577.     
  578.     def __init__(self, closeCallback = None):
  579.         super(AsyncSSLStream, self).__init__(closeCallback)
  580.         self.interruptedOperation = None
  581.  
  582.     
  583.     def openConnection(self, host, port, callback, errback, disableReadTimeout = None):
  584.         
  585.         def onSocketOpen(self):
  586.             self.socket.setblocking(1)
  587.             eventloop.callInThread(onSSLOpen, handleSSLError, socket.ssl, 'AsyncSSL onSocketOpen()', self.socket)
  588.  
  589.         
  590.         def onSSLOpen(ssl):
  591.             if self.socket is None:
  592.                 return None
  593.             
  594.             self.socket.setblocking(0)
  595.             self.ssl = ssl
  596.             callback(self)
  597.  
  598.         
  599.         def handleSSLError(error):
  600.             errback(SSLConnectionError())
  601.  
  602.         super(AsyncSSLStream, self).openConnection(host, port, onSocketOpen, errback, disableReadTimeout)
  603.  
  604.     
  605.     def resumeNormalCallbacks(self):
  606.         if self.readCallback is not None:
  607.             eventloop.addReadCallback(self.socket, self.onReadReady)
  608.         
  609.         if len(self.toSend) != 0:
  610.             eventloop.addWriteCallback(self.socket, self.onWriteReady)
  611.         
  612.  
  613.     
  614.     def handleSocketError(self, code, msg, operation):
  615.         if code in (socket.SSL_ERROR_WANT_READ, socket.SSL_ERROR_WANT_WRITE):
  616.             if self.interruptedOperation is None:
  617.                 self.interruptedOperation = operation
  618.             elif self.interruptedOperation != operation:
  619.                 util.failed('When talking to the network', details = 'socket error for the wrong SSL operation')
  620.                 self.closeConnection()
  621.                 return None
  622.             
  623.             eventloop.stopHandlingSocket(self.socket)
  624.             if code == socket.SSL_ERROR_WANT_READ:
  625.                 eventloop.addReadCallback(self.socket, self.onReadReady)
  626.             else:
  627.                 eventloop.addWriteCallback(self.socket, self.onWriteReady)
  628.         elif code in (socket.SSL_ERROR_ZERO_RETURN, socket.SSL_ERROR_SSL, socket.SSL_ERROR_SYSCALL, socket.SSL_ERROR_EOF):
  629.             self.handleEarlyClose(operation)
  630.         else:
  631.             super(AsyncSSLStream, self).handleSocketError(code, msg, operation)
  632.  
  633.     
  634.     def onWriteReady(self):
  635.         if self.interruptedOperation == 'read':
  636.             return self.onReadReady()
  637.         
  638.         
  639.         try:
  640.             if len(self.toSend) > 0:
  641.                 sent = self.ssl.write(self.toSend[0].data)
  642.             else:
  643.                 sent = 0
  644.         except socket.error:
  645.             (code, msg) = None
  646.             self.handleSocketError(code, msg, 'write')
  647.  
  648.         if self.interruptedOperation == 'write':
  649.             self.resumeNormalCallbacks()
  650.             self.interruptedOperation = None
  651.         
  652.         self.handleSentData(sent)
  653.  
  654.     
  655.     def onReadReady(self):
  656.         if self.interruptedOperation == 'write':
  657.             return self.onWriteReady()
  658.         
  659.         
  660.         try:
  661.             data = self.ssl.read(self.readSize)
  662.         except socket.error:
  663.             (code, msg) = None
  664.             self.handleSocketError(code, msg, 'read')
  665.  
  666.         if self.interruptedOperation == 'read':
  667.             self.resumeNormalCallbacks()
  668.             self.interruptedOperation = None
  669.         
  670.         self.handleReadData(data)
  671.  
  672.  
  673.  
  674. class ProxiedAsyncSSLStream(AsyncSSLStream):
  675.     
  676.     def openConnection(self, host, port, callback, errback, disableReadTimeout):
  677.         
  678.         def onSocketOpen(self):
  679.             self.socket.setblocking(1)
  680.             None(eventloop.callInThread, (onSSLOpen, handleSSLError), (lambda : openProxyConnection(self)), 'ProxiedAsyncSSL openProxyConnection()')
  681.  
  682.         
  683.         def openProxyConnection(self):
  684.             headers = {
  685.                 'User-Agent': '%s/%s (%s)' % (config.get(prefs.SHORT_APP_NAME), config.get(prefs.APP_VERSION), config.get(prefs.PROJECT_URL)),
  686.                 'Host': host }
  687.             if config.get(prefs.HTTP_PROXY_AUTHORIZATION_ACTIVE):
  688.                 username = config.get(prefs.HTTP_PROXY_AUTHORIZATION_USERNAME)
  689.                 password = config.get(prefs.HTTP_PROXY_AUTHORIZATION_PASSWORD)
  690.                 authString = username + ':' + password
  691.                 authString = b64encode(authString)
  692.                 headers['ProxyAuthorization'] = 'Basic ' + authString
  693.             
  694.             connectString = 'CONNECT %s:%d HTTP/1.1\r\n' % (host, port)
  695.             for header, value in headers.items():
  696.                 connectString += '%s: %s\r\n' % (header, value)
  697.             
  698.             connectString += '\r\n'
  699.             
  700.             try:
  701.                 self.socket.send(connectString)
  702.                 data = ''
  703.                 while data.find('\r\n\r\n') == -1:
  704.                     data += self.socket.recv(1)
  705.                 data = data.split('\r\n')
  706.                 if -1 == data[0].find(' 200 '):
  707.                     (None, eventloop.addIdle)((lambda : handleSSLError(NetworkError(data[0]))), 'Network Error')
  708.                 else:
  709.                     return socket.ssl(self.socket)
  710.             except socket.error:
  711.                 (code, msg) = None
  712.                 handleSSLError(msg)
  713.  
  714.  
  715.         
  716.         def onSSLOpen(ssl):
  717.             if self.socket is None or ssl is None:
  718.                 return None
  719.             
  720.             self.socket.setblocking(0)
  721.             self.ssl = ssl
  722.             callback(self)
  723.  
  724.         
  725.         def handleSSLError(error):
  726.             errback(SSLConnectionError())
  727.  
  728.         proxy_host = config.get(prefs.HTTP_PROXY_HOST)
  729.         proxy_port = config.get(prefs.HTTP_PROXY_PORT)
  730.         AsyncSocket.openConnection(self, proxy_host, proxy_port, onSocketOpen, errback, disableReadTimeout)
  731.  
  732.  
  733.  
  734. class ConnectionHandler(object):
  735.     """Base class to handle asynchronous network streams.  It implements a
  736.     simple state machine to deal with incomming data.
  737.  
  738.     Sending data: Use the sendData() method.
  739.  
  740.     Reading Data: Add entries to the state dictionary, which maps strings to
  741.     methods.  The state methods will be called when there is data available,
  742.     which can be read from the buffer variable.  The states dictionary can
  743.     contain a None value, to signal that the handler isn't interested in
  744.     reading at that point.  Use changeState() to switch states.
  745.  
  746.     Subclasses should override tho the handleClose() method to handle the
  747.     socket closing.
  748.     """
  749.     streamFactory = AsyncSocket
  750.     
  751.     def __init__(self):
  752.         self.buffer = NetworkBuffer()
  753.         self.states = {
  754.             'initializing': None,
  755.             'closed': None }
  756.         self.stream = self.streamFactory(closeCallback = self.closeCallback)
  757.         self.changeState('initializing')
  758.         self.name = ''
  759.  
  760.     
  761.     def __str__(self):
  762.         return '%s -- %s' % (self.__class__, self.state)
  763.  
  764.     
  765.     def openConnection(self, host, port, callback, errback, disableReadTimeout = None):
  766.         self.name = 'Outgoing %s:%s' % (host, port)
  767.         self.host = host
  768.         self.port = port
  769.         
  770.         def callbackIntercept(asyncSocket):
  771.             if callback:
  772.                 trapCall(self, callback, self)
  773.             
  774.  
  775.         self.stream.openConnection(host, port, callbackIntercept, errback, disableReadTimeout)
  776.  
  777.     
  778.     def closeConnection(self):
  779.         if self.stream.isOpen():
  780.             self.stream.closeConnection()
  781.         
  782.         self.changeState('closed')
  783.  
  784.     
  785.     def sendData(self, data, callback = None):
  786.         self.stream.sendData(data, callback)
  787.  
  788.     
  789.     def changeState(self, newState):
  790.         self.readHandler = self.states[newState]
  791.         self.state = newState
  792.         self.updateReadCallback()
  793.  
  794.     
  795.     def updateReadCallback(self):
  796.         if self.readHandler is not None:
  797.             self.stream.startReading(self.handleData)
  798.         elif self.stream.isOpen():
  799.             
  800.             try:
  801.                 self.stream.stopReading()
  802.             except KeyError:
  803.                 pass
  804.             except:
  805.                 None<EXCEPTION MATCH>KeyError
  806.             
  807.  
  808.         None<EXCEPTION MATCH>KeyError
  809.  
  810.     
  811.     def handleData(self, data):
  812.         self.buffer.addData(data)
  813.         lastState = self.state
  814.         self.readHandler()
  815.         while self.readHandler is not None and lastState != self.state:
  816.             lastState = self.state
  817.             self.readHandler()
  818.  
  819.     
  820.     def closeCallback(self, stream, type):
  821.         self.handleClose(type)
  822.  
  823.     
  824.     def handleClose(self, type):
  825.         '''Handle our stream becoming closed.  Type is either socket.SHUT_RD,
  826.         or socket.SHUT_WR.
  827.         '''
  828.         raise NotImplementedError()
  829.  
  830.  
  831.  
  832. class HTTPConnection(ConnectionHandler):
  833.     scheme = 'http'
  834.     
  835.     def __init__(self, closeCallback = None, readyCallback = None):
  836.         super(HTTPConnection, self).__init__()
  837.         self.shortVersion = 0
  838.         self.states['ready'] = None
  839.         self.states['response-status'] = self.onStatusData
  840.         self.states['response-headers'] = self.onHeaderData
  841.         self.states['response-body'] = self.onBodyData
  842.         self.states['chunk-size'] = self.onChunkSizeData
  843.         self.states['chunk-data'] = self.onChunkData
  844.         self.states['chunk-crlf'] = self.onChunkCRLFData
  845.         self.states['chunk-trailer'] = self.onChunkTrailerData
  846.         self.changeState('ready')
  847.         self.idleSince = clock()
  848.         self.unparsedHeaderLine = ''
  849.         self.pipelinedRequest = None
  850.         self.closeCallback = closeCallback
  851.         self.readyCallback = readyCallback
  852.         self.requestsFinished = 0
  853.         self.bytesRead = 0
  854.         self.sentReadyCallback = False
  855.         self.headerCallback = None
  856.         self.bodyDataCallback = None
  857.  
  858.     
  859.     def handleData(self, data):
  860.         self.bytesRead += len(data)
  861.         super(HTTPConnection, self).handleData(data)
  862.  
  863.     
  864.     def closeConnection(self):
  865.         super(HTTPConnection, self).closeConnection()
  866.         if self.closeCallback is not None:
  867.             self.closeCallback(self)
  868.             self.closeCallback = None
  869.         
  870.         self.checkPipelineNotStarted()
  871.  
  872.     
  873.     def checkPipelineNotStarted(self):
  874.         '''Call this when the connection is closed by Democracy or the other
  875.         side.  It will check if we have an unstarted pipeline request and 
  876.         send it the PipelinedRequestNeverStarted error
  877.         '''
  878.         if self.pipelinedRequest is not None:
  879.             errback = self.pipelinedRequest[1]
  880.             trapCall(self, errback, PipelinedRequestNeverStarted())
  881.             self.pipelinedRequest = None
  882.         
  883.  
  884.     
  885.     def canSendRequest(self):
  886.         if self.state == 'ready' and self.state != 'closed' and self.pipelinedRequest is None and not (self.willClose):
  887.             pass
  888.         return PIPELINING_ENABLED
  889.  
  890.     
  891.     def sendRequest(self, callback, errback, host, port, requestStartCallback = None, headerCallback = None, bodyDataCallback = None, method = 'GET', path = '/', headers = None, postVariables = None, postFiles = None):
  892.         """Sending an HTTP Request.  callback will be called if the request
  893.         completes normally, errback will be called if there is a network
  894.         error.
  895.  
  896.         Callback will be passed a dictionary that represents the HTTP
  897.         response,  it will have an entry for each header sent by the server as
  898.         well as as the following keys:
  899.             body, version, status, reason, method, path, host, port
  900.         They should be self explanatory, status and port will be integers, the
  901.         other items will be strings.
  902.  
  903.         If requestStartCallback is given, it will be called just before the
  904.         we start receiving data for the request (this can be a while after
  905.         sending the request in the case of pipelined requests).  It will be
  906.         passed this connection object.
  907.  
  908.         If headerCallback is given, it will be called when the headers are
  909.         read in.  It will be passed a response object whose body is set to
  910.         None.
  911.  
  912.         If bodyDataCallback is given it will be called as we read in the data
  913.         for the body.  Also, the connection won't store the body in memory,
  914.         and the callback is called, it will be passed None for the body.
  915.  
  916.         postVariables is a dictionary of variable names to values
  917.  
  918.         postFiles is a dictionary of variable names to dictionaries
  919.         containing filename, mimetype, and handle attributes. Handle
  920.         should be an already open file handle.
  921.         """
  922.         if not self.canSendRequest():
  923.             raise NetworkError(_('Unknown'), _('Internal Error: Not ready to send'))
  924.         
  925.         if headers is None:
  926.             headers = { }
  927.         else:
  928.             headers = headers.copy()
  929.         headers['Host'] = host.encode('idna')
  930.         if port != defaultPort(self.scheme):
  931.             headers['Host'] += ':%d' % port
  932.         
  933.         headers['Accept-Encoding'] = 'identity'
  934.         if method == 'POST' and postVariables is not None and len(postVariables) > 0 and postFiles is None:
  935.             postData = URLEncodeDict(postVariables)
  936.             headers['Content-Type'] = 'application/x-www-form-urlencoded'
  937.             headers['Content-Length'] = '%d' % len(postData)
  938.         elif method == 'POST' and postFiles is not None:
  939.             (postData, boundary) = multipartEncode(postVariables, postFiles)
  940.             headers['Content-Type'] = 'multipart/form-data; boundary=%s' % boundary
  941.             headers['Content-Length'] = '%d' % len(postData)
  942.         else:
  943.             postData = None
  944.         self.sendRequestData(method, path, headers, postData)
  945.         args = (callback, errback, requestStartCallback, headerCallback, bodyDataCallback, method, path, headers)
  946.         if self.state == 'ready':
  947.             self.startNewRequest(*args)
  948.         else:
  949.             self.pipelinedRequest = args
  950.  
  951.     
  952.     def startNewRequest(self, callback, errback, requestStartCallback, headerCallback, bodyDataCallback, method, path, headers):
  953.         """Called when we're ready to start processing a new request, either
  954.         because one has just been made, or because we've pipelined one, and
  955.         the previous request is done.
  956.         """
  957.         if requestStartCallback:
  958.             trapCall(self, requestStartCallback, self)
  959.             if self.state == 'closed':
  960.                 return None
  961.             
  962.         
  963.         self.callback = callback
  964.         self.errback = errback
  965.         self.headerCallback = headerCallback
  966.         self.bodyDataCallback = bodyDataCallback
  967.         self.method = method
  968.         self.path = path
  969.         self.requestHeaders = headers
  970.         self.headers = { }
  971.         self.contentLength = None
  972.         self.version = None
  973.         self.status = None
  974.         self.reason = None
  975.         self.bytesRead = 0
  976.         self.body = ''
  977.         self.willClose = True
  978.         self.chunked = False
  979.         self.chunks = []
  980.         self.idleSince = None
  981.         self.sentReadyCallback = False
  982.         self.changeState('response-status')
  983.  
  984.     
  985.     def sendRequestData(self, method, path, headers, data = None):
  986.         sendOut = []
  987.         path = path.encode('ascii', 'replace')
  988.         path = urllib.quote(path, safe = "-_.!~*'();/?:@&=+$,%#")
  989.         sendOut.append('%s %s HTTP/1.1\r\n' % (method, path))
  990.         for header, value in headers.items():
  991.             sendOut.append('%s: %s\r\n' % (header, value))
  992.         
  993.         sendOut.append('\r\n')
  994.         if data is not None:
  995.             sendOut.append(data)
  996.         
  997.         self.sendData(''.join(sendOut))
  998.  
  999.     
  1000.     def onStatusData(self):
  1001.         line = self.buffer.readline()
  1002.         if line is not None:
  1003.             self.handleStatusLine(line)
  1004.             if self.state == 'closed':
  1005.                 return None
  1006.             
  1007.             if self.shortVersion != 9:
  1008.                 self.changeState('response-headers')
  1009.             else:
  1010.                 self.startBody()
  1011.         
  1012.  
  1013.     
  1014.     def onHeaderData(self):
  1015.         while self.state == 'response-headers':
  1016.             line = self.buffer.readline()
  1017.             if line is None:
  1018.                 break
  1019.             
  1020.             self.handleHeaderLine(line)
  1021.  
  1022.     
  1023.     def onBodyData(self):
  1024.         if self.bodyDataCallback:
  1025.             if self.contentLength is None:
  1026.                 data = self.buffer.read()
  1027.             else:
  1028.                 bytesLeft = self.contentLength - self.bodyBytesRead
  1029.                 data = self.buffer.read(bytesLeft)
  1030.             if data == '':
  1031.                 return None
  1032.             
  1033.             self.bodyBytesRead += len(data)
  1034.             trapCall(self, self.bodyDataCallback, data)
  1035.             if self.state == 'closed':
  1036.                 return None
  1037.             
  1038.             if self.contentLength is not None and self.bodyBytesRead == self.contentLength:
  1039.                 self.finishRequest()
  1040.             
  1041.         elif self.contentLength is not None and self.buffer.length >= self.contentLength:
  1042.             self.body = self.buffer.read(self.contentLength)
  1043.             self.finishRequest()
  1044.         
  1045.  
  1046.     
  1047.     def onChunkSizeData(self):
  1048.         line = self.buffer.readline()
  1049.         if line is not None:
  1050.             sizeString = line.split(';', 1)[0]
  1051.             
  1052.             try:
  1053.                 self.chunkSize = int(sizeString, 16)
  1054.             except ValueError:
  1055.                 self.handleError(BadChunkSize(line))
  1056.                 return None
  1057.  
  1058.             if self.chunkSize != 0:
  1059.                 self.chunkBytesRead = 0
  1060.                 self.changeState('chunk-data')
  1061.             else:
  1062.                 self.changeState('chunk-trailer')
  1063.         
  1064.  
  1065.     
  1066.     def onChunkData(self):
  1067.         if self.bodyDataCallback:
  1068.             bytesLeft = self.chunkSize - self.chunkBytesRead
  1069.             data = self.buffer.read(bytesLeft)
  1070.             self.chunkBytesRead += len(data)
  1071.             if data == '':
  1072.                 return None
  1073.             
  1074.             trapCall(self, self.bodyDataCallback, data)
  1075.             if self.chunkBytesRead == self.chunkSize:
  1076.                 self.changeState('chunk-crlf')
  1077.             
  1078.         elif self.buffer.length >= self.chunkSize:
  1079.             self.chunks.append(self.buffer.read(self.chunkSize))
  1080.             self.changeState('chunk-crlf')
  1081.         
  1082.  
  1083.     
  1084.     def onChunkCRLFData(self):
  1085.         if self.buffer.length >= 2:
  1086.             crlf = self.buffer.read(2)
  1087.             if crlf != '\r\n':
  1088.                 self.handleError(CRLFExpected(crlf))
  1089.             else:
  1090.                 self.changeState('chunk-size')
  1091.         
  1092.  
  1093.     
  1094.     def onChunkTrailerData(self):
  1095.         line = self.buffer.readline()
  1096.         while line is not None:
  1097.             if line == '':
  1098.                 self.finishRequest()
  1099.                 break
  1100.             
  1101.             line = self.buffer.readline()
  1102.  
  1103.     
  1104.     def handleStatusLine(self, line):
  1105.         
  1106.         try:
  1107.             (version, status, reason) = line.split(None, 2)
  1108.         except ValueError:
  1109.             
  1110.             try:
  1111.                 (version, status) = line.split(None, 1)
  1112.                 reason = ''
  1113.             except ValueError:
  1114.                 version = ''
  1115.             except:
  1116.                 None<EXCEPTION MATCH>ValueError
  1117.             
  1118.  
  1119.             None<EXCEPTION MATCH>ValueError
  1120.  
  1121.         if not version.startswith('HTTP/'):
  1122.             self.buffer.unread(line + '\r\n')
  1123.             self.version = 'HTTP/0.9'
  1124.             self.status = 200
  1125.             self.reason = ''
  1126.             self.shortVersion = 9
  1127.         else:
  1128.             
  1129.             try:
  1130.                 status = int(status)
  1131.                 if status < 100 or status > 599:
  1132.                     self.handleError(BadStatusLine(line))
  1133.                     return None
  1134.             except ValueError:
  1135.                 self.handleError(BadStatusLine(line))
  1136.                 return None
  1137.  
  1138.             if version == 'HTTP/1.0':
  1139.                 self.shortVersion = 10
  1140.             elif version.startswith('HTTP/1.'):
  1141.                 self.shortVersion = 11
  1142.             else:
  1143.                 self.handleError(BadStatusLine(line))
  1144.                 return None
  1145.             self.version = version
  1146.             self.status = status
  1147.             self.reason = reason
  1148.  
  1149.     
  1150.     def handleHeaderLine(self, line):
  1151.         if self.unparsedHeaderLine == '':
  1152.             if line == '':
  1153.                 if self.status != 100:
  1154.                     self.startBody()
  1155.                 else:
  1156.                     self.changeState('response-status')
  1157.             elif ':' in line:
  1158.                 self.parseHeader(line)
  1159.             else:
  1160.                 self.unparsedHeaderLine = line
  1161.         elif len(line) > 0 and line[0] in (' ', '\t'):
  1162.             self.unparsedHeaderLine += line.lstrip()
  1163.             if ':' in self.unparsedHeaderLine:
  1164.                 self.parseHeader(self.unparsedHeaderLine)
  1165.                 self.unparsedHeaderLine = ''
  1166.             
  1167.         else:
  1168.             msg = 'line: %s, next line: %s' % (self.unparsedHeaderLine, line)
  1169.             self.handleError(BadHeaderLine(msg))
  1170.  
  1171.     
  1172.     def parseHeader(self, line):
  1173.         (header, value) = line.split(':', 1)
  1174.         value = value.strip()
  1175.         header = header.lstrip().lower()
  1176.         if value == '':
  1177.             print 'DTV: Warning: Bad Header from %s://%s:%s%s (%s)' % (self.scheme, self.host, self.port, self.path, line)
  1178.         
  1179.         if header not in self.headers:
  1180.             self.headers[header] = value
  1181.         else:
  1182.             self.headers[header] += ',%s' % value
  1183.  
  1184.     
  1185.     def startBody(self):
  1186.         self.findExpectedLength()
  1187.         self.checkChunked()
  1188.         self.decideWillClose()
  1189.         if self.headerCallback:
  1190.             trapCall(self, self.headerCallback, self.makeResponse())
  1191.         
  1192.         if self.state == 'closed':
  1193.             return None
  1194.         
  1195.         if self.status <= self.status:
  1196.             pass
  1197.         elif self.status <= 199 and self.status in (204, 304) and self.method == 'HEAD' or self.contentLength == 0:
  1198.             self.finishRequest()
  1199.         elif self.bodyDataCallback:
  1200.             self.bodyBytesRead = 0
  1201.         
  1202.         if not self.chunked:
  1203.             self.changeState('response-body')
  1204.         else:
  1205.             self.changeState('chunk-size')
  1206.         self.maybeSendReadyCallback()
  1207.  
  1208.     
  1209.     def checkChunked(self):
  1210.         te = self.headers.get('transfer-encoding', '')
  1211.         self.chunked = te.lower() == 'chunked'
  1212.  
  1213.     
  1214.     def findExpectedLength(self):
  1215.         self.contentLength = None
  1216.         if self.status == 416:
  1217.             
  1218.             try:
  1219.                 contentRange = self.headers['content-range']
  1220.             except KeyError:
  1221.                 pass
  1222.  
  1223.             m = re.search('bytes\\s+\\*/(\\d+)', contentRange)
  1224.             if m is not None:
  1225.                 
  1226.                 try:
  1227.                     self.contentLength = int(m.group(1))
  1228.                 except (ValueError, TypeError):
  1229.                     pass
  1230.                 except:
  1231.                     None<EXCEPTION MATCH>(ValueError, TypeError)
  1232.                 
  1233.  
  1234.             None<EXCEPTION MATCH>(ValueError, TypeError)
  1235.         
  1236.         if self.contentLength is None and self.headers.get('transfer-encoding') in ('identity', None):
  1237.             
  1238.             try:
  1239.                 self.contentLength = int(self.headers['content-length'])
  1240.             except (ValueError, KeyError):
  1241.                 pass
  1242.             except:
  1243.                 None<EXCEPTION MATCH>(ValueError, KeyError)
  1244.             
  1245.  
  1246.         None<EXCEPTION MATCH>(ValueError, KeyError)
  1247.         if self.contentLength < 0:
  1248.             self.contentLength = None
  1249.         
  1250.  
  1251.     
  1252.     def decideWillClose(self):
  1253.         if self.shortVersion != 11:
  1254.             self.willClose = True
  1255.         elif 'close' in self.headers.get('connection', '').lower():
  1256.             self.willClose = True
  1257.         elif not (self.chunked) and self.contentLength is None:
  1258.             self.willClose = True
  1259.         else:
  1260.             self.willClose = False
  1261.  
  1262.     
  1263.     def finishRequest(self):
  1264.         origCallback = self.callback
  1265.         if self.bodyDataCallback:
  1266.             body = None
  1267.         elif self.chunked:
  1268.             body = ''.join(self.chunks)
  1269.         else:
  1270.             body = self.body
  1271.         response = self.makeResponse(body)
  1272.         if self.stream.isOpen():
  1273.             if self.willClose:
  1274.                 self.closeConnection()
  1275.                 self.changeState('closed')
  1276.             elif self.pipelinedRequest is not None:
  1277.                 req = self.pipelinedRequest
  1278.                 self.pipelinedRequest = None
  1279.                 self.startNewRequest(*req)
  1280.             else:
  1281.                 self.changeState('ready')
  1282.                 self.idleSince = clock()
  1283.         
  1284.         trapCall(self, origCallback, response)
  1285.         self.requestsFinished += 1
  1286.         self.maybeSendReadyCallback()
  1287.  
  1288.     
  1289.     def makeResponse(self, body = None):
  1290.         response = self.headers.copy()
  1291.         response['body'] = body
  1292.         for key in ('version', 'status', 'reason', 'method', 'path', 'host', 'port', 'contentLength'):
  1293.             response[key] = getattr(self, key)
  1294.         
  1295.         return response
  1296.  
  1297.     
  1298.     def maybeSendReadyCallback(self):
  1299.         if self.readyCallback and self.canSendRequest() and not (self.sentReadyCallback):
  1300.             self.sentReadyCallback = True
  1301.             (eventloop.addIdle,)((lambda : self.readyCallback(self)), 'Ready Callback %s' % str(self))
  1302.         
  1303.  
  1304.     
  1305.     def handleClose(self, type):
  1306.         oldState = self.state
  1307.         self.closeConnection()
  1308.         if oldState == 'response-body' and self.contentLength is None:
  1309.             self.body = self.buffer.read()
  1310.             self.finishRequest()
  1311.         elif self.stream.timedOut:
  1312.             self.errback(ConnectionTimeout(self.host))
  1313.         else:
  1314.             self.errback(ServerClosedConnection(self.host))
  1315.         self.checkPipelineNotStarted()
  1316.  
  1317.     
  1318.     def handleError(self, error):
  1319.         self.closeConnection()
  1320.         trapCall(self, self.errback, error)
  1321.  
  1322.  
  1323.  
  1324. class HTTPSConnection(HTTPConnection):
  1325.     streamFactory = AsyncSSLStream
  1326.     scheme = 'https'
  1327.  
  1328.  
  1329. class ProxyHTTPSConnection(HTTPConnection):
  1330.     streamFactory = ProxiedAsyncSSLStream
  1331.     scheme = 'https'
  1332.  
  1333.  
  1334. class HTTPConnectionPool(object):
  1335.     '''Handle a pool of HTTP connections.
  1336.  
  1337.     We use the following stategy to handle new requests:
  1338.     * If there is an connection on the server that\'s ready to send, use that.
  1339.     * If we haven\'t hit our connection limits, create a new request
  1340.     * When a connection becomes closed, we look for our last 
  1341.  
  1342.     NOTE: "server" in this class means the combination of the scheme, hostname
  1343.     and port.
  1344.     '''
  1345.     HTTP_CONN = HTTPConnection
  1346.     HTTPS_CONN = HTTPSConnection
  1347.     PROXY_HTTPS_CONN = ProxyHTTPSConnection
  1348.     MAX_CONNECTIONS_PER_SERVER = 2
  1349.     CONNECTION_TIMEOUT = 300
  1350.     MAX_CONNECTIONS = 30
  1351.     
  1352.     def __init__(self):
  1353.         self.pendingRequests = []
  1354.         self.activeConnectionCount = 0
  1355.         self.freeConnectionCount = 0
  1356.         self.connections = { }
  1357.         eventloop.addTimeout(60, self.cleanupPool, 'Check HTTP Connection Timeouts')
  1358.  
  1359.     
  1360.     def _getServerConnections(self, scheme, host, port):
  1361.         key = '%s:%s:%s' % (scheme, host, port)
  1362.         
  1363.         try:
  1364.             return self.connections[key]
  1365.         except KeyError:
  1366.             self.connections[key] = {
  1367.                 'free': set(),
  1368.                 'active': set() }
  1369.             return self.connections[key]
  1370.  
  1371.  
  1372.     
  1373.     def _popPendingRequest(self):
  1374.         '''Try to choose a pending request to process.  If one is found,
  1375.         remove it from the pendingRequests list and return it.  If not, return
  1376.         None.
  1377.         '''
  1378.         if self.activeConnectionCount >= self.MAX_CONNECTIONS:
  1379.             return None
  1380.         
  1381.         for i in xrange(len(self.pendingRequests)):
  1382.             req = self.pendingRequests[i]
  1383.             if req['proxy_host']:
  1384.                 conns = self._getServerConnections(req['scheme'], req['proxy_host'], req['proxy_port'])
  1385.             else:
  1386.                 conns = self._getServerConnections(req['scheme'], req['host'], req['port'])
  1387.             if len(conns['free']) > 0 or len(conns['active']) < self.MAX_CONNECTIONS_PER_SERVER:
  1388.                 del self.pendingRequests[i]
  1389.                 return req
  1390.                 continue
  1391.         
  1392.         return None
  1393.  
  1394.     
  1395.     def _onConnectionClosed(self, conn):
  1396.         conns = self._getServerConnections(conn.scheme, conn.host, conn.port)
  1397.         if conn in conns['active']:
  1398.             conns['active'].remove(conn)
  1399.             self.activeConnectionCount -= 1
  1400.         elif conn in conns['free']:
  1401.             conns['free'].remove(conn)
  1402.             self.freeConnectionCount -= 1
  1403.         else:
  1404.             logging.warn('_onConnectionClosed called with connection not in either queue')
  1405.         self.runPendingRequests()
  1406.  
  1407.     
  1408.     def _onConnectionReady(self, conn):
  1409.         conns = self._getServerConnections(conn.scheme, conn.host, conn.port)
  1410.         if conn in conns['active']:
  1411.             conns['active'].remove(conn)
  1412.             self.activeConnectionCount -= 1
  1413.         else:
  1414.             logging.warn('_onConnectionReady called with connection not in the active queue')
  1415.         if conn not in conns['free']:
  1416.             conns['free'].add(conn)
  1417.             self.freeConnectionCount += 1
  1418.         else:
  1419.             logging.warn('_onConnectionReady called with connection already in the free queue')
  1420.         self.runPendingRequests()
  1421.  
  1422.     
  1423.     def addRequest(self, callback, errback, requestStartCallback, headerCallback, bodyDataCallback, url, method, headers, postVariables = None, postFiles = None):
  1424.         '''Add a request to be run.  The request will run immediately if we
  1425.         have a free connection, otherwise it will be queued.
  1426.  
  1427.         returns a request id that can be passed to cancelRequest
  1428.         '''
  1429.         proxy_host = None
  1430.         proxy_port = None
  1431.         (scheme, host, port, path) = parseURL(url)
  1432.         if scheme not in ('http', 'https') and host == '' or path == '':
  1433.             errback(MalformedURL(url))
  1434.             return None
  1435.         
  1436.         if scheme == 'http' and config.get(prefs.HTTP_PROXY_ACTIVE):
  1437.             if config.get(prefs.HTTP_PROXY_HOST) and config.get(prefs.HTTP_PROXY_PORT):
  1438.                 proxy_host = config.get(prefs.HTTP_PROXY_HOST)
  1439.                 proxy_port = config.get(prefs.HTTP_PROXY_PORT)
  1440.                 path = url
  1441.                 scheme = config.get(prefs.HTTP_PROXY_SCHEME)
  1442.                 if config.get(prefs.HTTP_PROXY_AUTHORIZATION_ACTIVE):
  1443.                     username = config.get(prefs.HTTP_PROXY_AUTHORIZATION_USERNAME)
  1444.                     password = config.get(prefs.HTTP_PROXY_AUTHORIZATION_PASSWORD)
  1445.                     authString = username + ':' + password
  1446.                     authString = b64encode(authString)
  1447.                     headers['ProxyAuthorization'] = 'Basic ' + authString
  1448.                 
  1449.             
  1450.         
  1451.         req = {
  1452.             'callback': callback,
  1453.             'errback': errback,
  1454.             'requestStartCallback': requestStartCallback,
  1455.             'headerCallback': headerCallback,
  1456.             'bodyDataCallback': bodyDataCallback,
  1457.             'scheme': scheme,
  1458.             'host': host,
  1459.             'port': port,
  1460.             'method': method,
  1461.             'path': path,
  1462.             'headers': headers,
  1463.             'postVariables': postVariables,
  1464.             'postFiles': postFiles,
  1465.             'proxy_host': proxy_host,
  1466.             'proxy_port': proxy_port }
  1467.         self.pendingRequests.append(req)
  1468.         self.runPendingRequests()
  1469.  
  1470.     
  1471.     def runPendingRequests(self):
  1472.         '''Find pending requests have a free connection, otherwise it will be
  1473.         queued.
  1474.         '''
  1475.         while True:
  1476.             req = self._popPendingRequest()
  1477.             if req is None:
  1478.                 return None
  1479.             
  1480.             if req['proxy_host']:
  1481.                 conns = self._getServerConnections(req['scheme'], req['proxy_host'], req['proxy_port'])
  1482.             else:
  1483.                 conns = self._getServerConnections(req['scheme'], req['host'], req['port'])
  1484.             if len(conns['free']) > 0:
  1485.                 conn = conns['free'].pop()
  1486.                 self.freeConnectionCount -= 1
  1487.                 conn.sendRequest(req['callback'], req['errback'], req['host'], req['port'], req['requestStartCallback'], req['headerCallback'], req['bodyDataCallback'], req['method'], req['path'], req['headers'], req['postVariables'], req['postFiles'])
  1488.             else:
  1489.                 conn = self._makeNewConnection(req)
  1490.             conns['active'].add(conn)
  1491.             self.activeConnectionCount += 1
  1492.             connectionCount = self.activeConnectionCount + self.freeConnectionCount
  1493.             if connectionCount > self.MAX_CONNECTIONS:
  1494.                 self._dropAFreeConnection()
  1495.                 continue
  1496.             self
  1497.  
  1498.     
  1499.     def _makeNewConnection(self, req):
  1500.         disableReadTimeout = req['postFiles'] is not None
  1501.         
  1502.         def openConnectionCallback(conn):
  1503.             conn.sendRequest(req['callback'], req['errback'], req['host'], req['port'], req['requestStartCallback'], req['headerCallback'], req['bodyDataCallback'], req['method'], req['path'], req['headers'], req['postVariables'], req['postFiles'])
  1504.  
  1505.         
  1506.         def openConnectionErrback(error):
  1507.             if req['proxy_host']:
  1508.                 conns = self._getServerConnections(req['scheme'], req['proxy_host'], req['proxy_port'])
  1509.             else:
  1510.                 conns = self._getServerConnections(req['scheme'], req['host'], req['port'])
  1511.             if conn in conns['active']:
  1512.                 conns['active'].remove(conn)
  1513.                 self.activeConnectionCount -= 1
  1514.             
  1515.             req['errback'](error)
  1516.  
  1517.         if req['scheme'] == 'https' and config.get(prefs.HTTP_PROXY_ACTIVE):
  1518.             connect_scheme = 'proxied_https'
  1519.         else:
  1520.             connect_scheme = req['scheme']
  1521.         if connect_scheme == 'http':
  1522.             conn = self.HTTP_CONN(self._onConnectionClosed, self._onConnectionReady)
  1523.         elif connect_scheme == 'https':
  1524.             conn = self.HTTPS_CONN(self._onConnectionClosed, self._onConnectionReady)
  1525.         elif connect_scheme == 'proxied_https':
  1526.             conn = self.PROXY_HTTPS_CONN(self._onConnectionClosed, self._onConnectionReady)
  1527.         else:
  1528.             raise AssertionError("Code shouldn't reach here. (connect scheme %s)" % connect_scheme)
  1529.         if req['proxy_host']:
  1530.             (None, None, None, None, eventloop.addIdle)((lambda : conn.openConnection(req['proxy_host'], req['proxy_port'], openConnectionCallback, openConnectionErrback, disableReadTimeout)), 'Open connection %s' % str(self))
  1531.         else:
  1532.             (None, None, None, None, eventloop.addIdle)((lambda : conn.openConnection(req['host'], req['port'], openConnectionCallback, openConnectionErrback, disableReadTimeout)), 'Open connection %s' % str(self))
  1533.         return conn
  1534.  
  1535.     
  1536.     def _dropAFreeConnection(self):
  1537.         firstTime = sys.maxint
  1538.         toDrop = None
  1539.         for conns in self.connections.values():
  1540.             for candidate in conns['free']:
  1541.                 if candidate.idleSince < firstTime:
  1542.                     toDrop = candidate
  1543.                     continue
  1544.             
  1545.         
  1546.         if toDrop is not None:
  1547.             toDrop.closeConnection()
  1548.         
  1549.  
  1550.     
  1551.     def cleanupPool(self):
  1552.         for serverKey in self.connections.keys():
  1553.             conns = self.connections[serverKey]
  1554.             toRemove = []
  1555.             for conn in conns['free']:
  1556.                 if conn.idleSince is not None and conn.idleSince + self.CONNECTION_TIMEOUT <= clock():
  1557.                     toRemove.append(conn)
  1558.                     continue
  1559.             
  1560.             for conn in toRemove:
  1561.                 conn.closeConnection()
  1562.             
  1563.             if len(conns['active']) == len(conns['active']):
  1564.                 pass
  1565.             elif len(conns['active']) == 0:
  1566.                 del self.connections[serverKey]
  1567.                 continue
  1568.         
  1569.         eventloop.addTimeout(60, self.cleanupPool, 'HTTP Connection Pool Cleanup')
  1570.  
  1571.  
  1572.  
  1573. class HTTPClient(object):
  1574.     '''High-level HTTP client object.  
  1575.     
  1576.     HTTPClients handle a single HTTP request, but may use several
  1577.     HTTPConnections if the server returns back with a redirection status code,
  1578.     asks for authorization, etc.  Connections are pooled using an
  1579.     HTTPConnectionPool object.
  1580.     '''
  1581.     connectionPool = HTTPConnectionPool()
  1582.     MAX_REDIRECTS = 10
  1583.     MAX_AUTH_ATTEMPS = 5
  1584.     
  1585.     def __init__(self, url, callback, errback, headerCallback = None, bodyDataCallback = None, method = 'GET', start = 0, etag = None, modified = None, cookies = None, postVariables = None, postFiles = None):
  1586.         if cookies == None:
  1587.             cookies = { }
  1588.         
  1589.         self.url = url
  1590.         self.callback = callback
  1591.         self.errback = errback
  1592.         self.headerCallback = headerCallback
  1593.         self.bodyDataCallback = bodyDataCallback
  1594.         self.method = method
  1595.         self.start = start
  1596.         self.etag = etag
  1597.         self.modified = modified
  1598.         self.cookies = cookies
  1599.         self.postVariables = postVariables
  1600.         self.postFiles = postFiles
  1601.         self.depth = 0
  1602.         self.authAttempts = 0
  1603.         self.updateURLOk = True
  1604.         self.originalURL = self.updatedURL = self.redirectedURL = url
  1605.         self.userAgent = '%s/%s (%s)' % (config.get(prefs.SHORT_APP_NAME), config.get(prefs.APP_VERSION), config.get(prefs.PROJECT_URL))
  1606.         self.connection = None
  1607.         self.cancelled = False
  1608.         self.initHeaders()
  1609.  
  1610.     
  1611.     def __str__(self):
  1612.         return '%s: %s' % (type(self).__name__, self.url)
  1613.  
  1614.     
  1615.     def cancel(self):
  1616.         self.cancelled = True
  1617.         if self.connection is not None:
  1618.             self.connection.closeConnection()
  1619.             self.connection = None
  1620.         
  1621.  
  1622.     
  1623.     def isValidCookie(self, cookie, scheme, host, port, path):
  1624.         if not time.time() - cookie['received'] < cookie['Max-Age'] and cookie['Version'] == '1' and self.hostMatches(host, cookie['Domain']) and path.startswith(cookie['Path']) and self.portMatches(str(port), cookie['Port']) and scheme == 'https':
  1625.             pass
  1626.         return not cookie['Secure']
  1627.  
  1628.     
  1629.     def dropStaleCookies(self):
  1630.         '''Remove cookies that have expired or are invalid for this URL'''
  1631.         (scheme, host, port, path) = parseURL(self.url)
  1632.         temp = { }
  1633.         for name in self.cookies:
  1634.             if self.isValidCookie(self.cookies[name], scheme, host, port, path):
  1635.                 temp[name] = self.cookies[name]
  1636.                 continue
  1637.         
  1638.         self.cookies = temp
  1639.  
  1640.     
  1641.     def hostMatches(self, host, host2):
  1642.         host = host.lower()
  1643.         host2 = host2.lower()
  1644.         if host.find('.') == -1:
  1645.             host = host + '.local'
  1646.         
  1647.         if host2.find('.') == -1:
  1648.             host2 = host2 + '.local'
  1649.         
  1650.         if host2.startswith('.'):
  1651.             return host.endswith(host2)
  1652.         else:
  1653.             return host == host2
  1654.  
  1655.     
  1656.     def portMatches(self, port, portlist):
  1657.         if portlist is None:
  1658.             return True
  1659.         
  1660.         portlist = portlist.replace(',', ' ').split()
  1661.         return port in portlist
  1662.  
  1663.     
  1664.     def initHeaders(self):
  1665.         self.headers = { }
  1666.         if self.start > 0:
  1667.             self.headers['Range'] = 'bytes=' + str(self.start) + '-'
  1668.         
  1669.         if self.etag is not None:
  1670.             self.headers['If-None-Match'] = self.etag
  1671.         
  1672.         if self.modified is not None:
  1673.             self.headers['If-Modified-Since'] = self.modified
  1674.         
  1675.         self.headers['User-Agent'] = self.userAgent
  1676.         self.setCookieHeader()
  1677.  
  1678.     
  1679.     def setCookieHeader(self):
  1680.         self.dropStaleCookies()
  1681.         if len(self.cookies) > 0:
  1682.             header = '$Version=1'
  1683.             for name in self.cookies:
  1684.                 header = '%s;%s=%s' % (header, name, self.cookies[name]['Value'])
  1685.                 if self.cookies[name].has_key('origPath'):
  1686.                     header = '%s;$Path=%s' % (header, self.cookies[name]['origPath'])
  1687.                 
  1688.                 if self.cookies[name].has_key('origDomain'):
  1689.                     header = '%s;$Domain=%s' % (header, self.cookies[name]['origDomain'])
  1690.                 
  1691.                 if self.cookies[name].has_key('origPort'):
  1692.                     header = '%s;$Port=%s' % (header, self.cookies[name]['origPort'])
  1693.                     continue
  1694.             
  1695.             self.headers['Cookie'] = header
  1696.         
  1697.  
  1698.     
  1699.     def startRequest(self):
  1700.         self.cancelled = False
  1701.         self.connection = None
  1702.         self.willHandleResponse = False
  1703.         self.gotBadStatusCode = False
  1704.         if 'Authorization' not in self.headers:
  1705.             (scheme, host, port, path) = parseURL(self.redirectedURL)
  1706.             
  1707.             def callback(authHeader):
  1708.                 if self.cancelled:
  1709.                     return None
  1710.                 
  1711.                 if authHeader is not None:
  1712.                     self.headers['Authorization'] = authHeader
  1713.                 
  1714.                 self.reallyStartRequest()
  1715.  
  1716.             httpauth.findHTTPAuth(callback, host.decode('ascii', 'replace'), path.decode('ascii', 'replace'))
  1717.         else:
  1718.             self.reallyStartRequest()
  1719.  
  1720.     
  1721.     def reallyStartRequest(self):
  1722.         if self.bodyDataCallback is not None:
  1723.             bodyDataCallback = self.onBodyData
  1724.         else:
  1725.             bodyDataCallback = None
  1726.         self.connectionPool.addRequest(self.callbackIntercept, self.errbackIntercept, self.onRequestStart, self.onHeaders, bodyDataCallback, self.url, self.method, self.headers, self.postVariables, self.postFiles)
  1727.  
  1728.     
  1729.     def statusCodeExpected(self, status):
  1730.         expectedStatusCodes = set([
  1731.             200])
  1732.         if self.start != 0:
  1733.             expectedStatusCodes.add(206)
  1734.         
  1735.         if self.etag is not None or self.modified is not None:
  1736.             expectedStatusCodes.add(304)
  1737.         
  1738.         return status in expectedStatusCodes
  1739.  
  1740.     
  1741.     def callbackIntercept(self, response):
  1742.         if self.cancelled:
  1743.             print 'WARNING: Callback on a cancelled request for %s' % self.url
  1744.             traceback.print_stack()
  1745.             return None
  1746.         
  1747.         if self.shouldRedirect(response):
  1748.             self.handleRedirect(response)
  1749.         elif self.shouldAuthorize(response):
  1750.             self.handleAuthorize(response)
  1751.         else:
  1752.             self.connection = None
  1753.             if not self.gotBadStatusCode:
  1754.                 if self.callback:
  1755.                     response = self.prepareResponse(response)
  1756.                     trapCall(self, self.callback, response)
  1757.                 
  1758.             elif self.errback:
  1759.                 error = UnexpectedStatusCode(response['status'])
  1760.                 self.errbackIntercept(error)
  1761.             
  1762.  
  1763.     
  1764.     def errbackIntercept(self, error):
  1765.         if self.cancelled:
  1766.             return None
  1767.         elif isinstance(error, PipelinedRequestNeverStarted):
  1768.             self.startRequest()
  1769.         elif isinstance(error, ServerClosedConnection) and self.connection is not None and self.connection.requestsFinished > 0 and self.connection.bytesRead == 0:
  1770.             self.startRequest()
  1771.         else:
  1772.             self.connection = None
  1773.             trapCall(self, self.errback, error)
  1774.  
  1775.     
  1776.     def onRequestStart(self, connection):
  1777.         if self.cancelled:
  1778.             connection.closeConnection()
  1779.         else:
  1780.             self.connection = connection
  1781.  
  1782.     
  1783.     def onHeaders(self, response):
  1784.         if self.shouldRedirect(response) or self.shouldAuthorize(response):
  1785.             self.willHandleResponse = True
  1786.         elif not self.statusCodeExpected(response['status']):
  1787.             self.gotBadStatusCode = True
  1788.         
  1789.         if self.headerCallback is not None:
  1790.             response = self.prepareResponse(response)
  1791.             if not trapCall(self, self.headerCallback, response):
  1792.                 self.cancel()
  1793.             
  1794.         
  1795.  
  1796.     
  1797.     def onBodyData(self, data):
  1798.         if not (self.willHandleResponse) and not (self.gotBadStatusCode) and self.bodyDataCallback:
  1799.             if not trapCall(self, self.bodyDataCallback, data):
  1800.                 self.cancel()
  1801.             
  1802.         
  1803.  
  1804.     
  1805.     def prepareResponse(self, response):
  1806.         response['original-url'] = self.originalURL
  1807.         response['updated-url'] = self.updatedURL
  1808.         response['redirected-url'] = self.redirectedURL
  1809.         response['filename'] = self.getFilenameFromResponse(response)
  1810.         response['charset'] = self.getCharsetFromResponse(response)
  1811.         
  1812.         try:
  1813.             response['cookies'] = self.getCookiesFromResponse(response)
  1814.         except:
  1815.             print 'ERROR in getCookiesFromResponse()'
  1816.             traceback.print_exc()
  1817.  
  1818.         return response
  1819.  
  1820.     
  1821.     def getCookiesFromResponse(self, response):
  1822.         '''Generates a cookie dictionary from headers in response
  1823.         '''
  1824.         
  1825.         def getAttrPair(attr):
  1826.             result = attr.strip().split('=', 1)
  1827.             if len(result) == 2:
  1828.                 (name, value) = result
  1829.             else:
  1830.                 name = result[0]
  1831.                 value = ''
  1832.             return (name, value)
  1833.  
  1834.         cookies = { }
  1835.         cookieStrings = []
  1836.         if response.has_key('set-cookie') or response.has_key('set-cookie2'):
  1837.             (scheme, host, port, path) = parseURL(self.redirectedURL)
  1838.             if response.has_key('set-cookie'):
  1839.                 cookieStrings.extend(response['set-cookie'].split(','))
  1840.             
  1841.             if response.has_key('set-cookie2'):
  1842.                 cookieStrings.extend(response['set-cookie2'].split(','))
  1843.             
  1844.             temp = []
  1845.             for string in cookieStrings:
  1846.                 if len(temp) > 0:
  1847.                     if temp[-1].count('"') % 2 == 1 and string.find('=') == -1 or string.find('=') > string.find(';'):
  1848.                         temp[-1] = '%s,%s' % (temp[-1], string)
  1849.                         continue
  1850.                 temp.append(string)
  1851.             
  1852.             cookieStrings = temp
  1853.             for string in cookieStrings:
  1854.                 string = string.strip()
  1855.                 pairs = string.split(';')
  1856.                 temp = []
  1857.                 for pair in pairs:
  1858.                     if len(temp) > 0 and temp[-1].count('"') % 2 == 1:
  1859.                         temp[-1] = '%s;%s' % (temp[-1], pair)
  1860.                         continue
  1861.                     temp.append(pair)
  1862.                 
  1863.                 pairs = temp
  1864.                 (name, value) = getAttrPair(pairs.pop(0))
  1865.                 cookie = {
  1866.                     'Value': value,
  1867.                     'Version': '1',
  1868.                     'received': time.time(),
  1869.                     'Path': '/'.join(path.split('/')[:-1]) + '/',
  1870.                     'Domain': host,
  1871.                     'Port': str(port),
  1872.                     'Secure': False }
  1873.                 for attr in pairs:
  1874.                     attr = attr.strip()
  1875.                     if attr.lower() == 'discard':
  1876.                         cookie['Discard'] = True
  1877.                         continue
  1878.                     if attr.lower() == 'secure':
  1879.                         cookie['Secure'] = True
  1880.                         continue
  1881.                     if attr.lower().startswith('version='):
  1882.                         cookie['Version'] = getAttrPair(attr)[1]
  1883.                         continue
  1884.                     if attr.lower().startswith('comment='):
  1885.                         cookie['Comment'] = getAttrPair(attr)[1]
  1886.                         continue
  1887.                     if attr.lower().startswith('commenturl='):
  1888.                         cookie['CommentURL'] = getAttrPair(attr)[1]
  1889.                         continue
  1890.                     if attr.lower().startswith('max-age='):
  1891.                         cookie['Max-Age'] = getAttrPair(attr)[1]
  1892.                         continue
  1893.                     if attr.lower().startswith('expires='):
  1894.                         now = time.time()
  1895.                         cookieval = getAttrPair(attr)[1].strip()
  1896.                         expires = get_cookie_expiration_date(cookieval)
  1897.                         expires -= time.timezone
  1898.                         if expires < now:
  1899.                             cookie['Max-Age'] = 0
  1900.                         else:
  1901.                             cookie['Max-Age'] = int(expires - now)
  1902.                     expires < now
  1903.                     if attr.lower().startswith('domain='):
  1904.                         cookie['origDomain'] = getAttrPair(attr)[1]
  1905.                         cookie['Domain'] = cookie['origDomain']
  1906.                         continue
  1907.                     if attr.lower().startswith('port='):
  1908.                         cookie['origPort'] = getAttrPair(attr)[1]
  1909.                         cookie['Port'] = cookie['origPort']
  1910.                         continue
  1911.                     if attr.lower().startswith('path='):
  1912.                         cookie['origPath'] = getAttrPair(attr)[1]
  1913.                         cookie['Path'] = cookie['origPath']
  1914.                         continue
  1915.                 
  1916.                 if not cookie.has_key('Discard'):
  1917.                     cookie['Discard'] = not cookie.has_key('Max-Age')
  1918.                 
  1919.                 if not cookie.has_key('Max-Age'):
  1920.                     cookie['Max-Age'] = str(1073741824)
  1921.                 
  1922.                 if self.isValidCookie(cookie, scheme, host, port, path):
  1923.                     cookies[name] = cookie
  1924.                     continue
  1925.             
  1926.         
  1927.         return cookies
  1928.  
  1929.     
  1930.     def findValueFromHeader(self, header, targetName):
  1931.         """Finds a value from a response header that uses key=value pairs with
  1932.         the ';' char as a separator.  This is how content-disposition and
  1933.         content-type work.
  1934.         """
  1935.         for part in header.split(';'):
  1936.             
  1937.             try:
  1938.                 (name, value) = part.split('=', 1)
  1939.             except ValueError:
  1940.                 continue
  1941.  
  1942.             if name.strip().lower() == targetName.lower():
  1943.                 return value.strip().strip('"')
  1944.                 continue
  1945.         
  1946.         return None
  1947.  
  1948.     
  1949.     def getFilenameFromResponse(self, response):
  1950.         
  1951.         try:
  1952.             disposition = response['content-disposition']
  1953.         except KeyError:
  1954.             pass
  1955.  
  1956.         filename = self.findValueFromHeader(disposition, 'filename')
  1957.         if filename is not None:
  1958.             return cleanFilename(filename)
  1959.         
  1960.         return filenameFromURL(util.unicodify(response['redirected-url']), clean = True)
  1961.  
  1962.     
  1963.     def getCharsetFromResponse(self, response):
  1964.         
  1965.         try:
  1966.             contentType = response['content-type']
  1967.         except KeyError:
  1968.             pass
  1969.  
  1970.         charset = self.findValueFromHeader(contentType, 'charset')
  1971.         if charset is not None:
  1972.             return charset
  1973.         
  1974.         return 'iso-8859-1'
  1975.  
  1976.     
  1977.     def shouldRedirect(self, response):
  1978.         if response['status'] in (301, 302, 303, 307) and self.depth < self.MAX_REDIRECTS:
  1979.             pass
  1980.         return 'location' in response
  1981.  
  1982.     
  1983.     def handleRedirect(self, response):
  1984.         self.depth += 1
  1985.         self.url = urljoin(self.url, response['location'])
  1986.         self.redirectedURL = self.url
  1987.         if response['status'] == 301 and self.updateURLOk:
  1988.             self.updatedURL = self.url
  1989.         else:
  1990.             self.updateURLOk = False
  1991.         if response['status'] == 303:
  1992.             self.method = 'GET'
  1993.             self.postVariables = None
  1994.         
  1995.         if 'Authorization' in self.headers:
  1996.             del self.headers['Authorization']
  1997.         
  1998.         self.startRequest()
  1999.  
  2000.     
  2001.     def shouldAuthorize(self, response):
  2002.         if response['status'] == 401 and self.authAttempts < self.MAX_AUTH_ATTEMPS:
  2003.             pass
  2004.         return 'www-authenticate' in response
  2005.  
  2006.     
  2007.     def handleAuthorize(self, response):
  2008.         match = re.search('(\\w+)\\s+realm\\s*=\\s*"(.*?)"$', response['www-authenticate'])
  2009.         if match is None:
  2010.             trapCall(self, self.errback, AuthorizationFailed())
  2011.             return None
  2012.         
  2013.         authScheme = unicode(match.expand('\\1'))
  2014.         realm = unicode(match.expand('\\2'))
  2015.         if authScheme.lower() != 'basic':
  2016.             trapCall(self, self.errback, AuthorizationFailed())
  2017.             return None
  2018.         
  2019.         
  2020.         def callback(authHeader):
  2021.             if authHeader is not None:
  2022.                 self.headers['Authorization'] = authHeader
  2023.                 self.authAttempts += 1
  2024.                 self.startRequest()
  2025.             else:
  2026.                 trapCall(self, self.errback, AuthorizationFailed())
  2027.  
  2028.         httpauth.askForHTTPAuth(callback, self.url, realm, authScheme)
  2029.  
  2030.  
  2031.  
  2032. def grabURL(url, callback, errback, headerCallback = None, bodyDataCallback = None, method = 'GET', start = 0, etag = None, modified = None, cookies = None, postVariables = None, postFiles = None, defaultMimeType = 'application/octet-stream', clientClass = HTTPClient):
  2033.     if cookies == None:
  2034.         cookies = { }
  2035.     
  2036.     if url.startswith('file://'):
  2037.         path = getFileURLPath(url)
  2038.         
  2039.         try:
  2040.             f = file(path)
  2041.         except EnvironmentError:
  2042.             errback(FileURLNotFoundError(path))
  2043.  
  2044.         
  2045.         try:
  2046.             data = f.read()
  2047.         except:
  2048.             errback(FileURLReadError(path))
  2049.  
  2050.         callback({
  2051.             'body': data,
  2052.             'updated-url': url,
  2053.             'redirected-url': url,
  2054.             'content-type': defaultMimeType })
  2055.     else:
  2056.         client = clientClass(url, callback, errback, headerCallback, bodyDataCallback, method, start, etag, modified, cookies, postVariables, postFiles)
  2057.         client.startRequest()
  2058.         return client
  2059.  
  2060.  
  2061. class HTTPHeaderGrabber(HTTPClient):
  2062.     '''Modified HTTPClient to get the headers for a URL.  It tries to do a
  2063.     HEAD request, then falls back on doing a GET request, and closing the
  2064.     connection right after the headers.
  2065.     '''
  2066.     
  2067.     def __init__(self, url, callback, errback):
  2068.         """HTTPHeaderGrabber support a lot less features than a real
  2069.         HTTPClient, mostly this is because they don't make sense in this
  2070.         context."""
  2071.         HTTPClient.__init__(self, url, callback, errback)
  2072.  
  2073.     
  2074.     def startRequest(self):
  2075.         self.method = 'HEAD'
  2076.         HTTPClient.startRequest(self)
  2077.  
  2078.     
  2079.     def errbackIntercept(self, error):
  2080.         if self.method == 'HEAD' and not (self.cancelled):
  2081.             self.method = 'GET'
  2082.             HTTPClient.startRequest(self)
  2083.         else:
  2084.             HTTPClient.errbackIntercept(self, error)
  2085.  
  2086.     
  2087.     def callbackIntercept(self, response):
  2088.         if self.method != 'GET' or self.willHandleResponse:
  2089.             HTTPClient.callbackIntercept(self, response)
  2090.         
  2091.  
  2092.     
  2093.     def onHeaders(self, headers):
  2094.         HTTPClient.onHeaders(self, headers)
  2095.         if self.method == 'GET' and not (self.willHandleResponse):
  2096.             headers['body'] = ''
  2097.             self.callback(self.prepareResponse(headers))
  2098.             self.cancel()
  2099.         
  2100.  
  2101.  
  2102.  
  2103. def grabHeaders(url, callback, errback, clientClass = HTTPHeaderGrabber):
  2104.     client = clientClass(url, callback, errback)
  2105.     client.startRequest()
  2106.     return client
  2107.  
  2108.